package org.wangbao.service;


import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

/**
 * <p>
 * 在线性时间内抽取主题类（新闻、博客等）网页的正文。
 * 采用了<b>基于行块分布函数</b>的方法，为保持通用性没有针对特定网站编写规则。
 * </p>
 * @author  Chen Xin
 * @version 1.0, 11/11/09
 */
public class TextExtract {
	
	private static List<String> lines  ;
	private final static int BL_BLOCK  ;
	private static int THRESHOLD ;
	private static String  html        ;
	private static boolean flag		   ;
	private static int 	   start	   ;
	private static int 	   end		   ;
	private static StringBuilder text  ;
	private static ArrayList<Integer> indexDistribution ;
	
	static {
		lines = new ArrayList<String>();
		indexDistribution = new ArrayList<Integer>();
		text = new StringBuilder();
		BL_BLOCK 	= 3;
		flag 		= false;
		/* 当待抽取的网页正文中遇到成块的新闻标题未剔除时，只要增大此阈值即可。*/
		/* 阈值增大，准确率提升，召回率下降；值变小，噪声会大，但可以保证抽到只有一句话的正文 */
		THRESHOLD	= 86;   
	}
	
	public static void setThreshold(int value) {
		THRESHOLD = value;
	}

	/**
	 * 抽取网页正文，不判断该网页是否是目录型。即已知传入的肯定是可以抽取正文的主题类网页。
	 * 
	 * @param _html 网页HTML字符串
	 * 
	 * @return 网页正文string
	 */
	public static String parse(String _html) {
		return parse(_html, false);
	}
	
	/**
	 * 判断传入HTML，若是主题类网页，则抽取正文；否则输出<b>"unkown"</b>。
	 * 
	 * @param _html 网页HTML字符串
	 * @param _flag true进行主题类判断, 省略此参数则默认为false
	 * 
	 * @return 网页正文string
	 */
	public static String parse(String _html, boolean _flag) {
		flag = _flag;
		html = _html;
		preProcess();
//		System.out.println(html);
		return getText();
	}
	
	private static void preProcess() {
		html = html.replaceAll("(?is)<!DOCTYPE.*?>", "");
		html = html.replaceAll("(?is)<!--.*?-->", "");				// remove html comment
		html = html.replaceAll("(?is)<script.*?>.*?</script>", ""); // remove javascript
		html = html.replaceAll("(?is)<style.*?>.*?</style>", "");   // remove css
		html = html.replaceAll("&.{2,5};|&#.{2,5};", " ");			// remove special char
		html = html.replaceAll("(?is)<.*?>", "");
		//"[^\u4E00-\u9FA50-9a-zA-Z_]"
		
		//<!--[if !IE]>|xGv00|9900d21eb16fa4350a3001b3974a9415<![endif]--> 
	}
	
	private static String getText() {
		lines = Arrays.asList(html.split("\n"));
		indexDistribution.clear();
		
		for (int i = 0; i < lines.size() - BL_BLOCK; i++) {
			int wordsNum = 0;
			for (int j = i; j < i + BL_BLOCK; j++) { 
				
				//.replaceAll("[^\u4E00-\u9FA50-9]", ""))
				//下一步算法：如果只包含数字，不包含中文则过滤整行
				//String tmp = lines.get(j);
				//if (tmp.matches("0-9a-zA-Z")&&!tmp.matches("[\u4E00-\u9FA5]"))
				//	continue;
				
				lines.set(j, lines.get(j).replaceAll("\\s+", "").replaceAll("[^\u4E00-\u9FA5\\n\\\r\\s\\t0-9a-zA-Z_，。、！？：“”（）…]", ""));
				wordsNum += lines.get(j).length();
			}
			indexDistribution.add(wordsNum);
			//System.out.println(wordsNum);
		}
		
		start = -1; end = -1;
		boolean boolstart = false, boolend = false;
		text.setLength(0);
		
		for (int i = 0; i < indexDistribution.size() - 1; i++) {
			if (indexDistribution.get(i) > THRESHOLD && ! boolstart) {
				if (indexDistribution.get(i+1).intValue()    != 0 
					|| indexDistribution.get(i+2).intValue() != 0
					|| indexDistribution.get(i+3).intValue() != 0) {
					boolstart = true;
					start = i;
					continue;
				}
			}
			if (boolstart) {
				if (indexDistribution.get(i).intValue() == 0 
					|| indexDistribution.get(i+1).intValue() == 0) {
					end = i;
					boolend = true;
				}
			}
			StringBuilder tmp = new StringBuilder();
			if (boolend) {
				//System.out.println(start+1 + "\t\t" + end+1);
				for (int ii = start; ii <= end; ii++) {
					if (lines.get(ii).length() < 5) continue;
					tmp.append(lines.get(ii) + "\n");
				}
				String str = tmp.toString();
				//System.out.println(str);
				//System.out.println("*******************************\n");
				if (str.contains("Copyright")  || str.contains("版权所有") 
					||str.contains("网友评论") || str.contains("热门图片") 
					||str.contains("相关阅读") || str.contains("我要评论") 
					||str.contains("每日推荐") || str.contains("关联阅读") ) continue; 
				text.append(str);
				boolstart = boolend = false;
			}
		}
		return text.toString();
	}
}
